"""
Created from Blagovest Taskov (taskov@gmail.com), Sofia Univiersity,
Bulgaria, 07.2010 for Python course final project.
None rights reserved.
"""
__author__ = 'taskov@gmail.com (Blagovest Taskov)'

import re
import codecs
import sys
import xml.dom.minidom
import os
import unittest
import time
import data_provider
from question import *
from constants import *

#all user settings are loaded form settings.txt and are saved in SETTINGS (map). If loading of these values fail,
#the program exits. Some of the values are name, type and I/O files settings, etc...
SETTINGS = {}

#the path for the questions indexes to be excluded. These file should contain information for file_source and question_number. Exmaple:
# my_first_questions.txt : 1-4, 7,8, 9 - 15
# this will exclude questions with indexes (starting counting from top and 0) 1,2,3,7,8,9,10,11,12,13,14 from my_first_questions.txt file
# if they exists.

def log_error(error_text):
    if PRINT_ERRORS:
        print(error_text)
    if LOG_ERRORS:
        try:
            file = open(LOG_FILE_NAME, 'a+')
        except:
            file.close()
            return
        today = time.localtime(time.time())
        time_str = '[' + str(today.tm_year) + '.' + str(today.tm_mon) + '.' + str(today.tm_mday) + '|' \
               + str(today.tm_hour) + ':' + str(today.tm_min) + ':' + str(today.tm_sec) + ']'
        file.write(time_str + ' ' + error_text + '\n')
    
#used for reading text files (for settings, filtration, plain text formatted questions)
#using utf-8 mainly (could be changed by changing ENCODING global var. Problems may occured in html parsing/codecs module)
def read_txt_file(file_name):
    if type(file_name).__name__ != 'str':
        log_error(ERROR_PREFIX + "Error occured while reading files. For information about work flow, please check the documentation" + ERROR_SUFFIX)
        exit(0)
    text = ''
    try:
        file = open(file_name, 'r', encoding = ENCODING)
        text = file.read()
    except(UnicodeEncodeError):
        log_error(ERROR_PREFIX + "This program supports utf-8 encoding only. Please, check if all the input files are text files in utf-8 encoding and try again" + ERROR_SUFFIX)
        return ''
    except:
        log_error(ERROR_PREFIX + "File " + file_name + " can not be opened and will be skiped from generating test data" + ERROR_SUFFIX)
    finally:
        try:
            file.close()
        except(UnboundLocalError):
            pass
    return text

#two questions should be separated with '\n\n'. Question are formatted in the way (on each line) :
# (1) Head, (2) Correct Answer (index, starting from 0), (3) Difficult, (4) .. (n) Possibly Answers
# String in the begging on the line to char '#' are considered as comments(labels) and are skipped (answer#42 is same as 42)
# Difficulty should be between 1 and 5 (its auto set to the closest valid value if its not)
def parse_txt_data(questions_data, file_name):
    questions_data.strip()
    questions = []
    questions_as_arr = questions_data.split('\n\n')
    index =  0
    for questions_as_str in questions_as_arr:
        question_parts = questions_as_str.split('\n')
        if len(question_parts) <= 1:
            continue
        #try splitting every string, to catch these with labels
        head_arr = question_parts[0].split(TXT_LABELS_SEPARATOR)
        if len(head_arr) == 1:
           head = question_parts[0]
        else:
           head = head_arr[1]

        correct_answer_arr = question_parts[1].split(TXT_LABELS_SEPARATOR)
        if len(correct_answer_arr) == 1:
           correct_answer_index = question_parts[1]
        else:
           correct_answer_index = correct_answer_arr[1]

        difficult_level_arr = question_parts[2].split(TXT_LABELS_SEPARATOR)
        if len(difficult_level_arr) == 1:
           difficult_level = question_parts[2]
        else:
           difficult_level = difficult_level_arr[1]
        answers = question_parts[3:]

        try:
            questions.append(question(head,answers,int(difficult_level), int(correct_answer_index), file_name, index))
            index = index + 1
        except(ValueError):
            log_error(ERROR_PREFIX + "Incorrect input parameters in text file " + file_name + ". Please check documentation for more information about input parameters" + ERROR_SUFFIX)
        except:
            log_error(ERROR_PREFIX + "ameters in text file" + file_name + ". Please check documentation" + ERROR_SUFFIX)
    return questions

def read_and_parse_txt_data(file_name):
    text = read_txt_file(file_name)
    try :
        data = parse_txt_data(text,file_name)
    except(IndexError,ValueError):
        log_error(ERROR_PREFIX + 'Unaible to parse file ' + file_name + '. This file will be ignored in generation. Check documentation for more information' + ERROR_SUFFIX)
        return []
    return data

#XML structure :
#<question>
#   <head> Tova e zaglavieto </head>
#   <difficult> 5 </difficult>
#   <correct> 2 </correct>
#   <answers>
#       <answer> 1 </answer>
#	...
#   </answers>
#</question>
# Difficulty should be between 1 and 5 (its auto set to the closest valid value if its not)
def read_and_parse_xml_data(file_name):
    questions = []
    if type(file_name).__name__ != 'str':
        log_error(ERROR_PREFIX + "Error occured while reading files. For information about work flow, please check the documentation" + ERROR_SUFFIX)
        exit(0)
    text = ''
    try:
        file = open(file_name, 'r', encoding = ENCODING)
        text = file.read()
    except(UnicodeEncodeError):
        log_error(ERROR_PREFIX + "This program supports utf-8 encoding only. Please, check if all the input files are text files in utf-8 encoding and try again" + ERROR_SUFFIX)
    except(IOError):
        log_error(ERROR_PREFIX + "Failed loading file " + file_name + '. Please try again later')
        return []
    try:
        xml_doc = xml.dom.minidom.parseString(text)
    except(xml.parsers.expat.ExpatError):
        log_error(ERROR_PREFIX + "XML file " + file_name + ' is ill-formatted and will be skipped during generation' + ERROR_SUFFIX)
        return []
    xml_doc_questions = xml_doc.getElementsByTagName('question')
    index = 0
    for item in xml_doc_questions:
        head = item.getElementsByTagName('head')[0].firstChild.data
        difficult_level = item.getElementsByTagName('difficult')[0].firstChild.data
        correct_answer_index = item.getElementsByTagName('correct')[0].firstChild.data

        answers = []
        all_answers_tree = item.getElementsByTagName('answers')[0].getElementsByTagName('answer')
        for answer_node in all_answers_tree:
           answers.append(answer_node.firstChild.data)
        try:
            questions.append(question(head, answers, difficult_level, correct_answer_index, file_name, index))
            index = index + 1
        except(ValueError):
            log_error(ERROR_PREFIX + "Incorrect input parameters. Please check documentation for more information about input parameters" + ERROR_SUFFIX)
        except:
            log_error(ERROR_PREFIX + "Unknown error occured during parsing input parameters from file " + file_name + ". Please check documentation" + ERROR_SUFFIX)
    return questions

#returns set from a to (b-1). parse_range(0,5) -> {0,1,2,3,4}
#Used to convert for exmaple, string 1-5 from the filters to set.
def parse_range(a,b):
    result = set()
    if a > b:
        return result
    while a < b:
        result.add(a)
        a = a + 1
    return result
#simplify strings. Allows to have mixed types of filters ( my_file.txt:1,2,4-6,9-11,19 )
#returns set of all needed indexes to be skipped (1,2,4-9) -> {1,2,4,5,6,7,8}
def extract_filter_indexes(text):
    filtered_indexes = text.split(',')
    result = set()
    for value in filtered_indexes:
        if value is None or value is '':
            continue
        arr = value.split('-')
        if len(arr) == 2:
            try:
                result.update(parse_range(int(arr[0]),int(arr[1])))
            except(ValueError):
                log_error(ERROR_PREFIX + "Input FILTER text file should contain only digits, and ',' , ':', '-' chars. For more information, please check the documentation" + ERROR_SUFFIX) 
                continue
        else:
            try:
                result.add(int(value))
            except(ValueError):
                log_error(ERROR_PREFIX + "Input FILTER text file should contain only digits, and ',' , ':', '-' chars. For more information, please check the documentation" + ERROR_SUFFIX) 
                continue
    return result

#reads the file containing information about needed question to be excluded, returns map.
#Keys are file names, values are sets of indexes. For example if we get map with key
#map["my_file.txt"] = {0,1,6,9}, that means that questions with these indexes (starting from 0 and top in the file)
#should be excluded in test generation
def get_needed_questions_indexes(file_name = None):
    if file_name is None:
        file_name = EXCLUDED_QUESTIONS_FILE_SOURCE
    try:
        files_filters_str = read_txt_file(file_name)
    except:
        return {}
    if files_filters_str == '' or files_filters_str is None:
        return {}
    files_filters = files_filters_str.split('\n')
    result = {}
    for filtered_file in files_filters:
        filter_data = filtered_file.split(':')
        filtered_file_name = os.curdir + '/' + SETTINGS['inputFilesPath'] + '/' + filter_data[0].strip()
        try:
            filtered_file_indexes = extract_filter_indexes(filter_data[1])
        except(IndexError):
            filtered_file_indexes = []
        result[filtered_file_name] = filtered_file_indexes
    return result

#gets list of questions, and returns only these, which should be considered in test generation (aka are not in the exlcluded
#from the user)
def filter_questions(questions):
    result = []
    filter_data = get_needed_questions_indexes()
    for each in questions:
        try:
            if not each.index in (filter_data['./' + each.source_file]):
                result.append(each)
        except(KeyError):
            result.append(each)
    return result

def get_restriction(file_name, source_file_name = None):
    if source_file_name is None:
        source_file_name = RESTRICTIONS
    try:
        restrictions_str = read_txt_file(source_file_name)
    except:
        return []
    restrictions_arr = restrictions_str.split('\n')
    
    if restrictions_arr == [] or restrictions_arr is None:
        return []
    for each in restrictions_arr:
        restrictions_data = each.split(':')
        if file_name == str(restrictions_data[0]):
            return restrictions_data[1].split(',')
    return []
# ******** region Parsing text to HTML

#little global var, used to trace question number in the html
question_number = 0

def setHeaders(matchObject):
    modified = matchObject.group()
    eqCount = 1
    while eqCount:
        if modified[eqCount] != '=':
            break
        eqCount += 1
    modified = modified[eqCount:len(modified) - eqCount+1]
    global question_number
    if question_number is not 0:
        question_number_str = str(question_number) + '. '
    else :
        question_number_str = ''
    modified='\n<h' + str(eqCount-1)+'>' + question_number_str + modified + '</h' + str(eqCount-1)+'>'
    question_number = question_number + 1
    return modified

def setLists(matchObject):
    modified = matchObject.group()
    modified = ('</p><ol type=a><li>'+modified)
    modified = re.sub(r'\n','</li>\n<li>',modified)
    modified = modified[0:len(modified) - 5]
    modified += '</ol>'
    modified = re.sub(r'\*(\s)?','',modified)
    modified += '<p>'
    return modified

def setNewLines(matchObject):
    return '<br/>'

def setParags(matchObject):
    modified = '<p></p>'
    return modified

def clearLinesInLists(matchObject):
    return '</li>'

def setParagHead(matchObject):
    modified = matchObject.group()
    return '</p>' + modified + '<p>'

#generates html-formmated string from string. Head-tags are generated form '=' (== meand <h2>*</h2>), <p>*</p> comes from /n/n.
def kiss(text):
    text = '<p>\n' + text + '</p>'
    text = re.sub(r'[\b\n]([=]{1,6})[^\=\n]*\1',setHeaders,text)
    text = '<p>' + text[4:]
    text = re.sub(r'<h[123456]>.*</h[123456]>',setParagHead,text)
    text = re.sub(r'([\*]+[^\n]*\n){2,}',setLists,text)
    text = re.sub(r'[\n]{2,}','</p><p>',text)
    text = re.sub(r'\n',setNewLines,text)
    text = re.sub(r'<p><br/>','<p>',text)
    text = re.sub(r'<br/></p>','</p>',text)
    text = re.sub(r'<p><br/></p>','',text)
    text = re.sub(r'<p></p>','',text)
    text = re.sub(r'</li><br/>',clearLinesInLists,text)
    return text
# endRegion parsing text to HTML

#adding settings.txt info in SETTINGS map. Critical, if loading fails program quits.
def load_settings():
    settings_txt = read_txt_file('settings.txt')
    settings_arr = settings_txt.split('\n')
    iterator = 0
    for line in settings_arr:
        try:
            if line is '' or line is None:
                continue 
            line = line.split(':')
            SETTINGS[line[0]] = line[1]
            
        except(KeyError,IndexError):
            continue
    try:
        SETTINGS['outputFileName'] = SETTINGS['outputFileName'].strip()
        SETTINGS['inputFilesPath'] = SETTINGS['inputFilesPath'].strip()
        SETTINGS['date'] = SETTINGS['date'].strip()
        SETTINGS['name'] = SETTINGS['name'].strip()
        SETTINGS['type'] = SETTINGS['type'].strip()
    except(ValueError):
        log_error(ERROR_PREFIX + "Failed loading settings file. Please check documentation for more information" + ERROR_SUFFIX)
        exit(0)

def save_html(html_text, file_name = None):
    if file_name is None:
        file_name = SETTINGS['outputFileName']
    print(html_text)
    try:
        file = codecs.open(file_name + '.html', 'w+', ENCODING)
        file.write(html_text)
    except:
        log_error(ERROR_PREFIX + "Failed saving html output file. Please try again later" + ERROR_SUFFIX)
    finally:
        file.close()
   
#generates string from list of questions, adds some html tags. Parses the string with kiss() and use save_html() to save it.
def process_test(test_data):
    test_data.sort(key = lambda question: question.difficult_level)
    text = ''
    for each in test_data:
        text = text + str(each) + '\n'
    try:
        html_txt = kiss( '===' + SETTINGS['name'] + '===' + '\n' + SETTINGS['date']+ '\n' + SETTINGS['type']+ '\n\n' + text) 
    except:
        log_error(ERROR_PREFIX + "Failed generating html page. Dont use html tags in question files data and check documentation for more information" + ERROR_SUFIX)
    save_html('<html><head><title>Test is created by BaiSelioSolutions software, open-source 1 person comunity, all rights \
                left :)</title><meta http-equiv="Content-Type" content="text/html; charset='+ ENCODING +'"/></head><body>' + html_txt\
                + '</body></html>')

# ********* region Finding question combination, satisfiyng user input params

#Recursive trying to get satisfiyng question combination. Tryes with list <data[0:'questionsCount'-1]>,
#and if its not satisfiyng reqs, begins to move indexes
#to 'right', until it doesnt find useful combination. So, for example we start to check indexes 0,1,2,3,4, if it doesn
#work, try 0,1,2,3,5, and so on to 0,1,2,3,...(questionCount-1). Then we process with the next : 0,1,2,4,(questionCount-1)
#untill we find useful combination or we reach the end (questionCount-4,questionCount-3,questionCount-2,questionCount-1)
def find_combination(questions, data, index_arr, movedIndex, from_difficulty, to_difficulty):
    if movedIndex == -1:
        return None
    else :
        for i in range(index_arr[movedIndex],index_arr[movedIndex + 1] + 1):
            index_arr[movedIndex] = i
            new_arr = get_arr_by_arr_indexes(data, index_arr)
            if arr_sum_is_between(new_arr, from_difficulty, to_difficulty):
                return get_arr_by_arr_indexes(questions, index_arr)
    find_combination(questions, data, index_arr, movedIndex - 1, from_difficulty, to_difficulty)

#data is sorted list, containg questions difficulties. Index_arr is used to move indexes of array, until we find
#satisfiyng combination. Its inited with first number form 0 to 'questionsCount' - 1, plus a fictive one
#which is the max posible valid value (used for succefful moving indexes. So, the last valid index ('questionsCount' - 1)
#will be 'moved' to the max posible value
def get_satisfing_combination(questions, questions_count, from_difficulty, to_difficulty):
    if from_difficulty > to_difficulty:
        return None
    data = []
    for each in questions:
        data.append(each.difficult_level)
    index_arr = []
    for i in range(0, questions_count):
        index_arr.append(i)
    index_arr.append(len(data)-1)
    return find_combination(questions, data, index_arr, questions_count - 1, from_difficulty, to_difficulty)

def arr_sum_is_between(arr,a,b):
    arr_sum = sum(arr)
    return arr_sum >= a and arr_sum <= b 

#returns array, created from <data>, getting only these values, which have their index in the <array>
#ATTENTION. Ignores last <array> value (caused due implementation detail)
def get_arr_by_arr_indexes(data, array):
    res = [] 
    for i in array:
        res.append(data[i])
    res = res[0:len(res)-1]
    return res
# end region Finding question combination, satisfiyng user input params

def start():
    load_settings()
    file_names = []
    test_questions = [] #list of list, containing questions grouped by their file source.
    try:
        for file in os.listdir(SETTINGS['inputFilesPath']):
            if file.endswith('.txt'):
                file_names.append(file)
                test_questions.extend([read_and_parse_txt_data(SETTINGS['inputFilesPath'] + '/' + file)])
            if file.endswith('.xml'):
                file_names.append(file)
                test_questions.extend([read_and_parse_xml_data(SETTINGS['inputFilesPath'] + '/' + file)])
    except(OSError):
        log_error(ERROR_PREFIX + "Failed loading input files. Probably there is something wrong with the dir name in settings.txt file" + ERROR_SUFFIX)
    
    added_questions = data_provider.get_extra_questions()
    #check if the provided data is okay
    if not type(added_questions).__name__ == 'list':
        log_error(ERROR_PREFIX + " data_provider.get_extra_questions() should provide list, of lists of questions. Wrong type provided. Extra questions will be skipped" +  ERROR_SUFFIX) 
        added_questions = []
    
    for each in added_questions:
        if not type(each).__name__ == 'list':
            log_error(ERROR_PREFIX + " data_provider.get_extra_questions() should provide list, of lists of questions. Wrong type provided. Extra questions will be skipped" +  ERROR_SUFFIX)
            added_questions = []
            break
        for q in each:
            if not type(q).__name__ == 'question':
                log_error(ERROR_PREFIX + " data_provider.get_extra_questions() should provide list, of lists of questions. Wrong type provided. Extra questions will be skipped" +  ERROR_SUFFIX)
                added_questions = []
                break

    #added questions if list of lists of questions, grouped by the data provider.
    #if restrictions for them are not met in the restrictions (text) file, all providet questions are added.s
    test_questions.extend(added_questions)

    iterator = 0
    
    result = []
    
    for questions in test_questions:
        test_questions[iterator] = filter_questions(test_questions[iterator])
        test_questions[iterator].sort(key = lambda q: q.difficult_level)

        try:
            restrictions = get_restriction(file_names[iterator])
        except:
            log_error(ERROR_PREFIX + 'Restrictions for extra provided questions from data container (or other) are missing. All questions will be added' + ERROR_SUFFIX)
            restrictions = [len(test_questions[iterator]),0,1000]
        try:
            test_questions[iterator] = get_satisfing_combination(test_questions[iterator], int(restrictions[0]), int(restrictions[1]), int(restrictions[2]))
        except:
            log_error(ERROR_PREFIX + 'RESTRICTIONS FILE FAILED VALIDATION (NOT ENOUGH QUESTIONS/IMPOSIBLE TO SATISFY REQS). PLEASE CHECK DOCUMENTATION, AND MAKE IT VALID')
            exit(0)
        if test_questions[iterator] is None:
            test_questions[iterator] = []
            log_error(ERROR_PREFIX + 'Failed test generation. Probably some of input data is wrong (or not enough). Check input data and settings.txt' + ERROR_SUFFIX)
        result.extend(test_questions[iterator])
        iterator = iterator + 1
    result.sort(key = lambda q: q.difficult_level)
    process_test(result)

